{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting Spark application\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<tr><th>ID</th><th>YARN Application ID</th><th>Kind</th><th>State</th><th>Spark UI</th><th>Driver log</th><th>Current session?</th></tr><tr><td>9</td><td>application_1561468620886_0011</td><td>pyspark</td><td>idle</td><td><a target=\"_blank\" href=\"http://hopsworks0.logicalclocks.com:8088/proxy/application_1561468620886_0011/\">Link</a></td><td><a target=\"_blank\" href=\"http://hopsworks0.logicalclocks.com:8042/node/containerlogs/container_e01_1561468620886_0011_01_000001/demo_deep_learning_admin000__meb10000\">Link</a></td><td>✔</td></tr></table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SparkSession available as 'spark'.\n"
     ]
    }
   ],
   "source": [
    "from hops import featurestore"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running sql: use demo_deep_learning_admin000_featurestore\n",
      "Logical query plan for getting 5 features from the featurestore created successfully\n",
      "SQL string for the query created successfully\n",
      "Running sql: SELECT avg_sold_for, avg_house_age, avg_house_size, avg_num_bidders, avg_house_worth FROM houses_sold_featuregroup_1 JOIN houses_for_sale_featuregroup_1 ON houses_sold_featuregroup_1.`area_id`=houses_for_sale_featuregroup_1.`area_id`\n",
      "Training Dataset created successfully"
     ]
    }
   ],
   "source": [
    "features_df = featurestore.get_features([\"avg_house_age\", \"avg_house_size\", \n",
    "                                         \"avg_house_worth\", \"avg_num_bidders\", \n",
    "                                         \"avg_sold_for\"])\n",
    "featurestore.create_training_dataset(\n",
    "    features_df, \"predict_house_sold_for_dataset\",\n",
    "    data_format=\"tfrecords\",\n",
    "    descriptive_statistics=False,\n",
    "    feature_correlation=False,\n",
    "    feature_histograms=False,\n",
    "    cluster_analysis=False\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "PySpark",
   "language": "",
   "name": "pysparkkernel"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "python",
    "version": 2
   },
   "mimetype": "text/x-python",
   "name": "pyspark",
   "pygments_lexer": "python2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
